#pragma once

#include "../ExternalTools/ExternalTools.h"

#include "../Loirey/loirey_GLOBAL.h"
#include "../Loirey/loirey_BasicStructure.h"
#include "../Loirey/loirey_SparseVector.h"

using namespace loirey;

class CDataSetForClassification
{
public:
	virtual ~CDataSetForClassification() { }
public:
	virtual int GetExampleAmount() = 0;
	virtual double GetExampleInitialWeight(int ExampleIndex) = 0;
	virtual int GetFeatureDimension() = 0;
	virtual double GetExampleFeature(int ExampleIndex, int FeatureIndex) = 0;
public:
	virtual void InitValidFeatures();
	virtual int GetValidFeatureAmount();
	virtual bool IsFeatureValid(int FeatureIndex);
	virtual void UpdateValidFeatures();
public:
	virtual double GetDataDensity();
	virtual double ComputeDataDensity();
	virtual void MakeSparseVector_ByExample(CSparseVector& DstFeatureVector, int SrcExampleIndex);
	virtual void MakeSparseVector_ByFeature(CSparseVector& DstFeatureVector, int SrcFeatureIndex);
};

class CSingleDataForClassification : public CDataSetForClassification
{
public:
	virtual ~CSingleDataForClassification() { }
	virtual int GetExampleAmount()
	{
		return 1;
	}
	virtual double GetExampleInitialWeight(int ExampleIndex)
	{
		return 1.0;
	}
	virtual int GetFeatureDimension() = 0;
	virtual double GetFeature(int FeatureIndex) = 0;
	virtual double GetExampleFeature(int ExampleIndex, int FeatureIndex)
	{
		if (ExampleIndex == 0)
			return GetFeature(FeatureIndex);
		else
			return 0.0;
	}
};

class CLabeledDataSetForClassification : public CDataSetForClassification
{
public:
	virtual ~CLabeledDataSetForClassification() { }
public:
	virtual int GetExampleLabel(int ExampleIndex) = 0;
};

class CWeightedClassificationExampleList;

class CLabeledDataSetForClassification_Proxy : public CLabeledDataSetForClassification
{
protected:
	CLabeledDataSetForClassification* _pSource;
	CLabeledDataSetForClassification* _pProxy;
public:
	int LastQuery;
	map<int, int> ProxyMap;
	map<int, int>::const_iterator itr;

public:
	CLabeledDataSetForClassification_Proxy();
	virtual ~CLabeledDataSetForClassification_Proxy();

	void InitSourceAndProxy(CLabeledDataSetForClassification* pSource, CLabeledDataSetForClassification* pProxy);
	void ClearProxyMap();
	void SetProxy(int ExampleIndexInSource, int ExampleIndexInProxy);

public:
	virtual int GetFeatureDimension();
	virtual int GetExampleAmount();
	virtual int GetExampleLabel(int ExampleIndex);
	virtual double GetExampleInitialWeight(int ExampleIndex);
	virtual double GetExampleFeature(int ExampleIndex, int FeatureIndex);
public:
	//virtual void InitValidFeatures();
	//virtual int GetValidFeatureAmount();
	//virtual bool IsFeatureValid(int FeatureIndex);
	//virtual void UpdateValidFeatures();
};

class CLabeledDataSetForClassification_ByExample : public CLabeledDataSetForClassification
{
protected:
	FILE* FIN;
	int _FreeSpacePosition;
	int _ExamplePoolSize;
	size_t _BaseOffset;
	size_t _DataBlockSize;
protected:
	size_t _FEATURE_POOL_SIZE_ScaleMB;
public:
	int ExampleAmount;
	int* rgExampleLabel;
	CSimpleTypeArray<double> InitialExampleWeightList;

	int FeatureDim;
	double* pFeatureValuePool;
	int* pExampleIndex;
	double** pExampleHeader;

public:
	static void GenerateFeatureFile(
		string strDstDataFileName,
		CLabeledDataSetForClassification* pDataSetByExample,
		size_t FEATURE_POOL_SIZE_ScaleMB
		);
	static void GenerateFeatureFile(
		string strDstDataFileName,
		CLabeledDataSetForClassification* pDataSetByExample,
		const CWeightedClassificationExampleList& ExampleList, bool fUseWeightInList,
		size_t FEATURE_POOL_SIZE_ScaleMB
		);
public:
	void myRelease();
	CLabeledDataSetForClassification_ByExample();
	virtual ~CLabeledDataSetForClassification_ByExample();

	void myInit(string strSrcDataFileName, size_t FEATURE_POOL_SIZE_ScaleMB);

	virtual int GetFeatureDimension();
	virtual int GetExampleAmount();
	virtual int GetExampleLabel(int ExampleIndex);
	virtual double GetExampleInitialWeight(int ExampleIndex);
	virtual double GetExampleFeature(int ExampleIndex, int FeatureIndex);
};

class CLabeledDataSetForClassification_ByFeature : public CLabeledDataSetForClassification
{
protected:
	FILE* FIN;
	int _FreeSpacePosition;
	int _FeaturePoolSize;
	size_t _BaseOffset;
	size_t _DataBlockSize;
protected:
	size_t _FEATURE_POOL_SIZE_ScaleMB;
public:
	int ExampleAmount;
	int* rgExampleLabel;
	CSimpleTypeArray<double> InitialExampleWeightList;

	int FeatureDim;
	double* pFeatureValuePool;
	int* pFeatureIndex;
	double** pFeatureHeader;

public:
	static void GenerateFeatureFile(
		string strDstDataFileName,
		CLabeledDataSetForClassification* pDataSetByExample,
		size_t FEATURE_POOL_SIZE_ScaleMB
		);
	static void GenerateFeatureFile(
		string strDstDataFileName,
		CLabeledDataSetForClassification* pDataSetByExample,
		const CWeightedClassificationExampleList& ExampleList, bool fUseWeightInList,
		size_t FEATURE_POOL_SIZE_ScaleMB
		);
public:
	void myRelease();
	CLabeledDataSetForClassification_ByFeature();
	virtual ~CLabeledDataSetForClassification_ByFeature();

	void myInit(string strSrcDataFileName, size_t FEATURE_POOL_SIZE_ScaleMB);

	virtual int GetFeatureDimension();
	virtual int GetExampleAmount();
	virtual int GetExampleLabel(int ExampleIndex);
	virtual double GetExampleInitialWeight(int ExampleIndex);
	virtual double GetExampleFeature(int ExampleIndex, int FeatureIndex);
};

class CLabeledDataSetForClassification_ByGroupedFeature : public CLabeledDataSetForClassification
{
protected:
	FILE* FIN;
	size_t _BaseOffset;
	size_t _DataBlockSize;
public:
	int ExampleAmount, FeatureDim;
	CSimpleTypeArray<int> LabelList;
	CSimpleTypeArray<double> InitialWeightList;
protected:
	CSimpleTypeArray<int> fea_index;
	CSimpleTypeArray<double*> fv_headers;
	CSimpleTypeArray<int> fea2grp;

	CSimpleTypeArray<double> fv_data;

	int grp_amount;
	int per_grp_size, grp_pool_size;
	int in_cache_fea_amount;
	CSimpleTypeArray<int> grp_headers;
	CSimpleTypeArray<int> grp_length;
	CSimpleTypeArray<int> grp2cache;
	CSimpleTypeArray<int> cache2grp;

public:
	static void GenerateRandomFeatureIndex(CSimpleTypeArray<int>& DstRFI, CLabeledDataSetForClassification* pDataSet);
	static void GenerateFeatureFile(
		string strDstDataFileName,
		CLabeledDataSetForClassification* pDataSetByExample,
		const CSimpleTypeArray<int>& RandomFeatureIndex,
		size_t FEATURE_POOL_SIZE_ScaleMB
		);
	static void GenerateFeatureFile(
		string strDstDataFileName,
		CLabeledDataSetForClassification* pDataSetByExample,
		const CSimpleTypeArray<int>& RandomFeatureIndex,
		const CWeightedClassificationExampleList& ExampleList, bool fUseWeightInList,
		size_t FEATURE_POOL_SIZE_ScaleMB
		);
protected:
	void myRelease();
	int RandomlySelectUncachedGroup();
	void LoadGroup(int GroupIndex, int CachePosition);
public:
	CLabeledDataSetForClassification_ByGroupedFeature();
	virtual ~CLabeledDataSetForClassification_ByGroupedFeature();

	void myInit(string strSrcDataFileName, int GroupAmount, size_t FEATURE_POOL_SIZE_ScaleMB);

	virtual int GetFeatureDimension();
	virtual int GetExampleAmount();
	virtual int GetExampleLabel(int ExampleIndex);
	virtual double GetExampleInitialWeight(int ExampleIndex);
	virtual double GetExampleFeature(int ExampleIndex, int FeatureIndex);

	virtual void InitValidFeatures();
	virtual int GetValidFeatureAmount();
	virtual bool IsFeatureValid(int FeatureIndex);
	virtual void UpdateValidFeatures();
};

class C_LDS_Sparse_ByExample : public CLabeledDataSetForClassification
{
public:
	class CExampleHeader
	{
	public:
		int Label;
		size_t BlockOffset;
		double InitialWeight;
	};

public:
	class CDataFileWriter
	{
	protected:
		FILE* FOUT;
		size_t LastBlockOffset;
		CExampleHeader tExampleHeader;
		CSparseVector tSparseVector;
	public:
		int FeatureDimension;

	public:
		CDataFileWriter();
		virtual ~CDataFileWriter();
		virtual void myInit(string strFN_DstData, int FeatureDimension);
		virtual void Finish();
		virtual void PushBack(const CSparseVector& FeatureVector, int Label, double InitialWeight);
		virtual void ModifyFeatureDimension(int NewFeatureDimension);
	};

protected:
	FILE* FIN;
	size_t IVPairCachePoolSize;
	size_t CachedIVPairAmount;
public:
	int ExampleAmount, FeatureDimension;
	int CachedExampleAmount;
	CSimpleTypeArray<CExampleHeader> HeaderList;
	CSimpleTypeArray<int> Example2CacheList;
	CSimpleTypeArray<int> Cache2ExampleList;
	CSimpleTypeArray<CSparseVector> CachedDataList;
public:
	CExampleHeader* pExampleHeader;
	int* pExample2Cache;
	int* pCache2Example;
	CSparseVector* pCachedData;

public:
	C_LDS_Sparse_ByExample();
	virtual ~C_LDS_Sparse_ByExample();
	virtual void myRelease();
	virtual void Clear();
public:
	virtual void myInit(string strFN_Data, size_t FEATURE_POOL_SIZE_ScaleMB);
	virtual void RemoveCachedExample(int CachedIndex);
	virtual void NewCachedExample(int ExampleIndex);
	virtual int GetExampleCacheIndex(int ExampleIndex);
	virtual const CSparseVector& GetExampleFeatureSparseVector(int ExampleIndex);
public:
	virtual int GetExampleAmount();
	virtual double GetExampleInitialWeight(int ExampleIndex);
	virtual int GetFeatureDimension();
	virtual double GetExampleFeature(int ExampleIndex, int FeatureIndex);
public:
	virtual int GetExampleLabel(int ExampleIndex);
public:
	virtual double GetDataDensity();
	virtual void MakeSparseVector_ByExample(CSparseVector& DstFeatureVector, int SrcExampleIndex);
};

class CLabeledDataSetForClassification_InMemory : public CLabeledDataSetForClassification
{
public:
	int ExampleAmount;
	int FeatureDimension;
	CSimpleTypeArray<double> FeatureList;
	CSimpleTypeArray<int> LabelList;
	CSimpleTypeArray<double> WeightList;

public:
	virtual ~CLabeledDataSetForClassification_InMemory() { }
public:
	virtual void myInit_ByFeature(string strFN_DataByFeature, size_t FEATURE_POOL_SIZE_ScaleMB);
	virtual void myInit_ByFeature(CLabeledDataSetForClassification_ByFeature& DataSetByFeature);

	//static void GenerateDataFile_ByExample(CLabeledDataSetForClassification* pDataSet, string strFN_DstData);
	//virtual void myInit_ByExample(string strFN_Data);
	virtual void myInit_ByExample(CLabeledDataSetForClassification* pDataSet);

	virtual void ClearAndInit(int FeatureDim);
	virtual void PushBack(int Label, const double* Feature, double InitialWeight = 1.0);

	virtual int GetFeatureDimension();
	virtual int GetExampleAmount();
	virtual int GetExampleLabel(int ExampleIndex);
	virtual double GetExampleInitialWeight(int ExampleIndex);
	virtual double GetExampleFeature(int ExampleIndex, int FeatureIndex);
};

class CWeightedClassificationExample
{
public:
	int ExampleIndex;
	double Weight;

public:
	CWeightedClassificationExample();
	CWeightedClassificationExample(int ExampleIndex, double Weight = 1.0);
	static bool CompareByExampleIndex(const CWeightedClassificationExample& a, const CWeightedClassificationExample& b);
	static bool CompareByWeight(const CWeightedClassificationExample& a, const CWeightedClassificationExample& b);
};

class CWeightedClassificationExampleList
{
public:
	int ExampleAmount;
	double TotalExampleWeight;
	vector<CWeightedClassificationExample> rgExample;

public:
	CWeightedClassificationExampleList();
public:
	virtual void myInit(CDataSetForClassification* pDataSet, bool fUseUniformedWeight = true);
	virtual void myInit(int ExampleAmount);
	virtual void myInit(const CWeightedClassificationExampleList& SourceExampleList, double SampleRate = 1.0);
	virtual void myInit(const CWeightedClassificationExampleList& SourceExampleList, int TargetExampleAmount);
public:
	virtual void SortByIndex();
	virtual void SortByWeight();
	virtual void RandomSplit(CSimpleTypeArray<CWeightedClassificationExampleList>& DstSubExampleList) const;
	virtual void RandomSplit(CSimpleTypeArray<CWeightedClassificationExampleList>& DstSubExampleList, int SubListAmount) const;
	virtual void RandomSplit(
		CWeightedClassificationExampleList& DstEL_A,
		CWeightedClassificationExampleList& DstEL_B,
		double EL_A_Ratio);
	virtual void RandomSplit(
		CWeightedClassificationExampleList& DstEL_A,
		CWeightedClassificationExampleList& DstEL_B,
		int EL_A_Size);
	virtual void AddEquals(const CWeightedClassificationExampleList& Another);
public:
	virtual void Clear();
	virtual void PushBack(const CWeightedClassificationExample& Example);
	virtual void PushBack(int ExampleIndex, double Weight = 1.0);
	virtual void ComputeTotalExampleWeight();
	virtual void InitializeExampleWeightFromDataSet(CDataSetForClassification* pDataSet);
public:
	virtual void MultiplyEachWeightBy(double Ratio);
	virtual void NormalizeWeight(double TargetTotalExampleWeight = 1.0);
};

